package utils;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;

import org.apache.commons.lang3.StringUtils;

/**
 * 
 * @author victorclf
 * @author pedroyossis
 */
public class TwitterParseUtil {
        private static final String HTML_TAG_START = "<a";
        private static final String HTML_TAG_END = "</a>";
        
        public static String preformatBlock(String token) {
                token = token.toLowerCase();
                token = removeHTMLTags(token);
                token = removeSpecialCharTags(token);
                token = removeWeirdCharacters(token);
                token = trimToken(token);
                token = removeExtraSpaces(token);
                return token;
        }
        
        public static String preformatToken(String token) {
                token = trimToken(token);
                token = removeExtraSpaces(token);
                return token;
        }
        
        public static boolean isTrivialWord(String token) {
                return (token.length() <= 2 && StringUtils.isAlpha(token))
                                || Constants.getTrivialWordSet().contains(token);
        }

        public static boolean isValidWord(String token) {
                return token != null && !token.trim().isEmpty();
        }
        
        public static List<String> tokenize(String line) {
                StringTokenizer tokenizer = new StringTokenizer(line);
                List<String> tokens = new LinkedList<String>();
                
                while (tokenizer.hasMoreTokens()) {
                        tokens.add(tokenizer.nextToken());
                }
                
                return tokens;
        }
        
        public static List<String> getValidTokens(List<String> rawTokens) {
                List<String> validTokens = new ArrayList<String>();
                
                for (String token : rawTokens) {
                        token = preformatToken(token);
                        
                        if (TwitterParseUtil.isValidWord(token) && !TwitterParseUtil.isTrivialWord(token)) {
                                validTokens.add(token);
                        }
                }
                
                return validTokens;
        }

        private static String removeHTMLTags(String line) {
                while (true) {
                        int startPos = line.indexOf(HTML_TAG_START);
                        if (startPos < 0) break;
                                
                        int endPos = line.indexOf(HTML_TAG_END, startPos + 1);
                        
                        if (endPos < 0) endPos = line.length();
                        else endPos += HTML_TAG_END.length();

                        line = line.substring(0, startPos) + line.substring(endPos);
                }
                
                return line;
        }
        
        private static String removeSpecialCharTags(String line) {
                return line.replaceAll("&[a-z0-9;]*;", "");
        }

        private static String trimToken(String token) {
                return StringUtils.strip(token, " -");
        }

        private static String removeExtraSpaces(String token) {
                return token.replaceAll(" {2,}", " ");
        }

        private static String removeWeirdCharacters(String token) {
                return token.replaceAll("[^\\p{L}0-9\\- ]", " ");
        }
}